def jupyter_settings():
%matplotlib inline
plt.style.use( 'bmh' )
plt.rcParams['figure.figsize'] = [25, 12]
plt.rcParams['font.size'] = 24
display( HTML( '<style>.container { width:100% !important; }</style>') )
sns.set()
def cramer_v(x, y):
cm = pd.crosstab(x, y).values
n = cm.sum()
r, k = cm.shape
chi2 = stats.chi2_contingency( cm )[0]
chi2corr = max(0, chi2 - (k - 1) * (r - 1) / (n - 1) )
kcorr = k - (k - 1) ** 2 / (n - 1)
rcorr = r - (r - 1) ** 2 / (n - 1)
return np.sqrt((chi2corr / n) / ( min(kcorr - 1, rcorr - 1 )))
def cross_validation(X, y, model,model_name = "Model", test_size = 0.3, cv = 10, top_k = 2000, verbose = True):
list_scores = []
for i in range(1, cv + 1):
X_train, X_test, y_train, y_test = train_test_split( X, y, test_size = test_size )
# Preprocess data
X_train, X_test, y_train, y_test = preprocessing(X_train, X_test, y_train, y_test)
# Model training
model = model.fit(X_train, y_train)
# Model evaluation
scores = print_scores(y_test = y_test,
y_pred = model.predict(X_test),
predict_proba = model.predict_proba(X_test),
top_k = top_k,
verbose = verbose)
list_scores.append(scores)
list_scores = np.array(list_scores).transpose()
data_frame = pd.DataFrame( {'Model Name': model_name,
'Precision Top-K CV': np.round( np.mean( list_scores[0] ), 2 ).astype(str ) + ' +/- ' + np.round( np.std( list_scores[0] ), 4 ).astype( str ),
'Recall Top-K CV': np.round( np.mean( list_scores[1] ), 2 ).astype( str ) + ' +/- ' + np.round( np.std( list_scores[1] ), 4 ).astype( str ),
'F1 Score CV': np.round( np.mean( list_scores[2] ), 2 ).astype( str ) + ' +/- ' + np.round( np.std( list_scores[2] ), 4 ).astype( str ),
'AUC Score CV': np.round( np.mean( list_scores[3] ), 2 ).astype( str ) + ' +/- ' + np.round( np.std( list_scores[3] ), 4 ).astype( str )}, index=[0] )
return data_frame
def plot_curves(y_test, predict_proba, verbose):
fpr, tpr, _ = roc_curve(y_test.values, predict_proba[:,1])
auc = np.round(roc_auc_score(y_test, predict_proba[::,1]),3)
auc = auc
if verbose == True:
# plt.figure(figsize = (4, 4))
# plt.plot(fpr,tpr)
# plt.ylabel('True Positive Rate')
# plt.xlabel('False Positive Rate')
# plt.title('ROC Curve | AUC Score: ' + str( auc ))
plt.figure(figsize = (4, 4))
skplt.metrics.plot_cumulative_gain(y_test, predict_proba, figsize = (4,4))
return auc
def score_top_k(y_test, predict_proba, top_k):
df = pd.DataFrame()
df['predictions'] = predict_proba[:,1]
df['y_test'] = y_test.values
df = df.sort_values('predictions', ascending = False).reset_index(drop = True).reset_index()
df['score'] = df.apply(lambda x: 1 if ( x['index'] <= top_k and x['y_test'] == 1) else 0, axis = 1 )
precision = df['score'].sum() / top_k
recall = df['score'].sum() / df['y_test'].sum()
return {'precision': precision, 'recall': recall}
def print_scores(y_test, y_pred, predict_proba, top_k,verbose):
precision = np.round(precision_score(y_test, y_pred, pos_label = 1),3)
recall = np.round(recall_score (y_test, y_pred, pos_label = 1),3)
f1 = np.round(f1_score (y_test, y_pred, pos_label = 1),3)
auc = plot_curves(y_test, predict_proba, verbose)
precision_top_k = score_top_k(y_test, predict_proba, top_k)['precision']
recall_top_k = score_top_k(y_test, predict_proba, top_k)['recall']
if verbose == True:
print("Precision top K: ", precision_top_k, " | Recall Top K: ", recall_top_k, f1, " | AUC Score: ", auc)
print("Confusion Matrix:\n", confusion_matrix(y_test, y_pred))
scores = [precision_top_k, recall_top_k, f1, auc]
return scores
def preprocessing(X_train, X_test, y_train, y_test):
# Transformation
# Gender
X_train['gender'] = X_train['gender'].apply(lambda x: 0 if x == 'Male' else 1)
X_test['gender'] = X_test['gender'].apply(lambda x: 0 if x == 'Male' else 1)
# Vehicle damage
X_train['vehicle_damage'] = X_train['vehicle_damage'].apply(lambda x: 0 if x == 'No' else 1)
X_test['vehicle_damage'] = X_test['vehicle_damage'].apply(lambda x: 0 if x == 'No' else 1)
# Vehicle age
X_train['vehicle_age'] = X_train['vehicle_age'].apply(lambda x: 0 if x == '< 1 Year' else 1 if x == '1-2 Year' else 2)
X_test['vehicle_age'] = X_test['vehicle_age'].apply(lambda x: 0 if x == '< 1 Year' else 1 if x == '1-2 Year' else 2)
# region_code - Frequency Encoding / Target Encoding / Weighted Target Encoding
# Target Encoding
aux = X_train.copy()
aux['response'] = y_train
target_encode_region_code = aux.groupby('region_code')['response'].mean()
X_train['region_code'] = X_train['region_code'].map( target_encode_region_code )
X_test['region_code'] = X_test['region_code'].map( target_encode_region_code )
X_test['region_code'].fillna(0, inplace = True)
# policy sales channel - Frequency Encoding / Target Encoding / Weighted Target Encoding
# Frequency encoding
target_encode_policy_sales = aux.groupby('policy_sales_channel')['response'].count() / len(insurance_data5)
X_train['policy_sales_channel'] = X_train['policy_sales_channel'].map( target_encode_policy_sales )
X_test['policy_sales_channel'] = X_test['policy_sales_channel'].map( target_encode_policy_sales )
X_test['policy_sales_channel'].fillna(0, inplace = True)
# Feature selection
X_train = X_train[['age', 'region_code', 'policy_sales_channel', 'previously_insured', 'annual_premium', 'vintage', 'vehicle_damage']]
X_test = X_test[['age', 'region_code', 'policy_sales_channel', 'previously_insured', 'annual_premium', 'vintage', 'vehicle_damage']]
return X_train, X_test, y_train, y_test
jupyter_settings()
